Skip to content

Commit

Permalink
integrate chains sheet into crawler, handle exceptional cases and upd…
Browse files Browse the repository at this point in the history
…ate test cases (#37)

* Add COUNT_API_KEY to environment variables, refactor Discord and GitHub functions, and update test cases

* Refactor YouTube video date retrieval to use YouTube Data API and improve schema validation

* Add COUNT_API_KEY to .env.example for environment configuration

* Refactor environment configuration to remove COUNT_API_KEY and update YouTube channel info function to use imported COUNT_API_KEY

* Refactor YouTube functions to use YouTube Data API, improve schema validation, and memoize channel info retrieval

* Refactor main function to restore YouTube metrics retrieval and clean up commented code

* Refactor imports to use local ofetch module instead of external ofetch package

* Update getTelegramGroupChatInfo to return memberCount and onlineCount

* Remove ignoreResponseError option from getGithubRepositoryRelease function

* Add null check for response in getGithubRepositoryRelease function
octave08 authored Dec 14, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent ac58229 commit 75baf2c
Showing 61 changed files with 647 additions and 388 deletions.
2 changes: 1 addition & 1 deletion src/functions/coinmarketcap/getMetadataFromCMC.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import { ofetch } from "@/lib/ofetch"
import { getDocument } from "@/utils/getDocument"
import { githubOrgURLSchema, githubRepoURLSchema } from "@/validators/github"
import { subredditURLSchema } from "@/validators/reddit"
import { telegramURLSchema } from "@/validators/telegram"
import { entries, filter, first, fromEntries, map, pipe } from "@fxts/core"
import { destr } from "destr"
import { ofetch } from "ofetch"

export const getMetadataFromCMC = async (slug: string) => {
const html = await ofetch(`https://coinmarketcap.com/currencies/${slug}`, {
4 changes: 2 additions & 2 deletions src/functions/discord/getDiscordInviteInfo.test.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { getDiscordInviteInfo } from "@/functions/discord/getDiscordInviteInfo"
import { describe, expect, it, vi } from "vitest"

describe("getDiscordInviteInfo function format tests", () => {
it("should match the expected response format", async () => {
describe("getDiscordInviteInfo", () => {
it("solana", async () => {
const inviteId = "solana"

const response = await getDiscordInviteInfo(inviteId)
2 changes: 1 addition & 1 deletion src/functions/discord/getDiscordInviteInfo.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { ofetch } from "@/lib/ofetch"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"
import { z } from "zod"

/*
4 changes: 2 additions & 2 deletions src/functions/discord/getMemberCountFromDiscord.ts
Original file line number Diff line number Diff line change
@@ -3,9 +3,9 @@ import { getLastSegment } from "@/utils/getLastSegment"
import { discordURLSchema } from "@/validators/discord"

export const getMemberCountFromDiscord = async (discordUrl: string) => {
const validUrl = discordURLSchema.parse(discordUrl)
const isValid = discordURLSchema.parse(discordUrl)

const inviteId = getLastSegment(validUrl)
const inviteId = getLastSegment(isValid)
const res = await getDiscordInviteInfo(inviteId)

const { approximate_member_count } = res
2 changes: 1 addition & 1 deletion src/functions/github/getClosedIssueCountFromGithub.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { env } from "@/lib/env"
import { ofetch } from "ofetch"
import { ofetch } from "@/lib/ofetch"

export const getClosedIssueCountFromGithub = async (
githubRepositoryLink: string,
2 changes: 1 addition & 1 deletion src/functions/github/getClosedPRCountFromGithub.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { env } from "@/lib/env"
import { ofetch } from "ofetch"
import { ofetch } from "@/lib/ofetch"

export const getClosedPRCountFromGithub = async (
githubRepositoryLink: string,
2 changes: 1 addition & 1 deletion src/functions/github/getGithubOrgFromWebsite.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { ofetch } from "@/lib/ofetch"
import { getDocument } from "@/utils/getDocument"
import { filter, map, pipe, toArray, uniq } from "@fxts/core"
import { ofetch } from "ofetch"

export const getGithubOrgFromWebsite = async (url: string) => {
const html = await ofetch<string>(url, { parseResponse: (txt) => txt })
12 changes: 10 additions & 2 deletions src/functions/github/getGithubOrganizationInfo.test.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,20 @@
import { getGithubOrganizationInfo } from "@/functions/github/getGithubOrganizationInfo"
import { describe, expect, it, vi } from "vitest"

describe("getGithubOrganizationInfo function format tests", () => {
it("should match the expected response format", async () => {
describe("getGithubOrganizationInfo", () => {
it("base", async () => {
const organizationName = "base-org"

const response = await getGithubOrganizationInfo(organizationName)

expect(response.login).toBe(organizationName)
})

it("bitcoin", async () => {
const organizationName = "bitcoin"

const response = await getGithubOrganizationInfo(organizationName)

expect(response.login).toBe(organizationName)
})
})
4 changes: 2 additions & 2 deletions src/functions/github/getGithubOrganizationInfo.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { env } from "@/lib/env"
import { ofetch } from "@/lib/ofetch"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"
import type { aC } from "vitest/dist/chunks/reporters.D7Jzd9GS.js"
import { z } from "zod"

@@ -25,7 +25,7 @@ export const getGithubOrganizationInfo = memoize(

export const getGithubOrganizationInfoSchema = z.object({
login: z.string(),
name: z.string(),
name: z.string().nullable(),
public_repos: z.number(),
followers: z.number(),
type: z.string(),
2 changes: 1 addition & 1 deletion src/functions/github/getGithubRepositoryBranch.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { env } from "@/lib/env"
import { ofetch } from "@/lib/ofetch"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"
import { z } from "zod"

/*
2 changes: 1 addition & 1 deletion src/functions/github/getGithubRepositoryContributors.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { env } from "@/lib/env"
import { ofetch } from "@/lib/ofetch"
import {
flat,
map,
@@ -9,7 +10,6 @@ import {
toArray,
toAsync,
} from "@fxts/core"
import { ofetch } from "ofetch"
import { z } from "zod"

/*
2 changes: 1 addition & 1 deletion src/functions/github/getGithubRepositoryInfo.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { env } from "@/lib/env"
import { ofetch } from "@/lib/ofetch"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"
import { z } from "zod"

/*
16 changes: 14 additions & 2 deletions src/functions/github/getGithubRepositoryRelease.test.ts
Original file line number Diff line number Diff line change
@@ -2,7 +2,7 @@ import { getGithubRepositoryRelease } from "@/functions/github/getGithubReposito
import { describe, expect, it, vi } from "vitest"

describe("getGithubRepositoryRelease function format tests", () => {
it("should match the expected response format", async () => {
it("base", async () => {
const organizationName = "base-org"
const repositoryName = "node"

@@ -11,6 +11,18 @@ describe("getGithubRepositoryRelease function format tests", () => {
repositoryName,
)

expect(response.id).toBeGreaterThan(0)
expect(response).toBeTruthy()
})

it("cardano", async () => {
const organizationName = "cardano-foundation"
const repositoryName = "CIPs"

const response = await getGithubRepositoryRelease(
organizationName,
repositoryName,
)

expect(response).toBeNull()
})
})
6 changes: 5 additions & 1 deletion src/functions/github/getGithubRepositoryRelease.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { env } from "@/lib/env"
import { ofetch } from "@/lib/ofetch"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"
import { z } from "zod"

/*
@@ -18,6 +18,10 @@ export const getGithubRepositoryRelease = memoize(
},
)

if (!response) {
return null
}

return getGithubRepositoryReleaseSchema.parse(response)
},
)
3 changes: 1 addition & 2 deletions src/functions/github/getLastCommitDateFromGithub.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import { getGithubRepositoryBranch } from "@/functions/github/getGithubRepositoryBranch"
import { getGithubRepositoryInfo } from "@/functions/github/getGithubRepositoryInfo"
import { getLastSegment } from "@/utils/getLastSegment"

export const getLastCommitDateFromGithub = async (
githubRepositoryLink: string,
@@ -20,5 +19,5 @@ export const getLastCommitDateFromGithub = async (
default_branch,
)

return lastCommitDate
return new Date(lastCommitDate).toISOString()
}
4 changes: 2 additions & 2 deletions src/functions/github/getLatestReleaseDateFromGithub.ts
Original file line number Diff line number Diff line change
@@ -7,10 +7,10 @@ export const getLatestReleaseDateFromGithub = async (
.split("/")
.reverse()

const { published_at } = await getGithubRepositoryRelease(
const data = await getGithubRepositoryRelease(
organizationName,
repositoryName,
)

return published_at
return data ? new Date(data.published_at).toISOString() : null
}
4 changes: 2 additions & 2 deletions src/functions/github/getLatestReleaseNameFromGithub.ts
Original file line number Diff line number Diff line change
@@ -7,10 +7,10 @@ export const getLatestReleaseNameFromGithub = async (
.split("/")
.reverse()

const { name } = await getGithubRepositoryRelease(
const data = await getGithubRepositoryRelease(
organizationName,
repositoryName,
)

return name
return data ? data.name : null
}
2 changes: 1 addition & 1 deletion src/functions/github/getOpenIssueCountFromGithub.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { env } from "@/lib/env"
import { ofetch } from "ofetch"
import { ofetch } from "@/lib/ofetch"

export const getOpenIssueCountFromGithub = async (
githubRepositoryLink: string,
2 changes: 1 addition & 1 deletion src/functions/github/getOpenPRCountFromGithub.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { env } from "@/lib/env"
import { ofetch } from "ofetch"
import { ofetch } from "@/lib/ofetch"

export const getOpenPRCountFromGithub = async (
githubRepositoryLink: string,
2 changes: 1 addition & 1 deletion src/functions/github/getTotalIssueCountFromGithub.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { env } from "@/lib/env"
import { ofetch } from "ofetch"
import { ofetch } from "@/lib/ofetch"

export const getTotalIssueCountFromGithub = async (
githubRepositoryLink: string,
2 changes: 0 additions & 2 deletions src/functions/github/getTotalPRCountFromGithub.test.ts
Original file line number Diff line number Diff line change
@@ -7,8 +7,6 @@ describe("getTotalPRCountFromGithub", () => {

const response = await getTotalPRCountFromGithub(githubRepositoryUrl)

console.log({ response })

expect(response).toBeGreaterThan(0)
})
})
2 changes: 1 addition & 1 deletion src/functions/github/getTotalPRCountFromGithub.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { env } from "@/lib/env"
import { ofetch } from "ofetch"
import { ofetch } from "@/lib/ofetch"

export const getTotalPRCountFromGithub = async (
githubRepositoryLink: string,
2 changes: 1 addition & 1 deletion src/functions/npm/getLastReleaseDateFromNpm.ts
Original file line number Diff line number Diff line change
@@ -8,5 +8,5 @@ export const getLastReleaseDateFromNpm = async (npmLink: string) => {

const data = await getNpmPackageInfo(packageName)

return data.time.modified
return new Date(data.time.modified).toISOString()
}
2 changes: 1 addition & 1 deletion src/functions/npm/getNpmLastDayDownloads.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { ofetch } from "@/lib/ofetch"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"
import { z } from "zod"

/*
2 changes: 1 addition & 1 deletion src/functions/npm/getNpmLastWeekDownloads.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { ofetch } from "@/lib/ofetch"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"
import { z } from "zod"

/*
2 changes: 1 addition & 1 deletion src/functions/npm/getNpmPackageInfo.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { ofetch } from "@/lib/ofetch"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"
import { z } from "zod"

/*
2 changes: 1 addition & 1 deletion src/functions/reddit/getRedditAccessToken.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { REDDIT_CLIENT_ID, env } from "@/lib/env"
import { ofetch } from "@/lib/ofetch"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"
import { z } from "zod"

/*
2 changes: 1 addition & 1 deletion src/functions/reddit/getRedditCommunityInfo.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { getRedditAccessToken } from "@/functions/reddit/getRedditAccessToken"
import { ofetch } from "@/lib/ofetch"
import { defaultBrowserHeaders } from "@/utils/defaultBrowserHeaders"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"
import { z } from "zod"

/*
2 changes: 1 addition & 1 deletion src/functions/reddit/getRedditCommunityLastPostDate.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { PROXY_URL } from "@/lib/env"
import { ofetch } from "@/lib/ofetch"
import { getDocument } from "@/utils/getDocument"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"

export const getRedditCommunityLastPostDate = memoize(async (slug: string) => {
const html = await ofetch(`${PROXY_URL}/https://reddit.com/r/${slug}/new`, {
8 changes: 7 additions & 1 deletion src/functions/reddit/getRedditCommunityMemberCount.test.ts
Original file line number Diff line number Diff line change
@@ -2,9 +2,15 @@ import { getRedditCommunityMemberCount } from "@/functions/reddit/getRedditCommu
import { describe, expect, it } from "vitest"

describe("getRedditCommunityMemberCount", () => {
it("solana", async () => {
it.skip("solana", async () => {
const memberCount = await getRedditCommunityMemberCount("solana")

expect(memberCount).toBeGreaterThan(0)
})

it("polkadot", async () => {
const memberCount = await getRedditCommunityMemberCount("Polkadot")

expect(memberCount).toBeGreaterThan(0)
})
})
4 changes: 2 additions & 2 deletions src/functions/reddit/getRedditCommunityMemberCount.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { PROXY_URL } from "@/lib/env"
import { ofetch } from "@/lib/ofetch"
import { getDocument } from "@/utils/getDocument"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"

export const getRedditCommunityMemberCount = memoize(async (slug: string) => {
const html = await ofetch(`${PROXY_URL}/https://reddit.com/r/${slug}`, {
@@ -15,7 +15,7 @@ export const getRedditCommunityMemberCount = memoize(async (slug: string) => {
)

if (!targetH1 || !targetH1.parentElement) {
throw new Error('No <h1> tag with "r/solana" found.')
throw new Error(`No <h1> tag with "r/${slug}" found.`)
}

const siblingWithMembers = Array.from(targetH1.parentElement.children).find(
2 changes: 1 addition & 1 deletion src/functions/reddit/getRedditCommunityNewPosts.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { getRedditAccessToken } from "@/functions/reddit/getRedditAccessToken"
import { ofetch } from "@/lib/ofetch"
import { defaultBrowserHeaders } from "@/utils/defaultBrowserHeaders"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"
import { z } from "zod"

/*
2 changes: 1 addition & 1 deletion src/functions/sim/getSimInfo.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { ofetch } from "@/lib/ofetch"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"
import { z } from "zod"

/*
18 changes: 14 additions & 4 deletions src/functions/telegram/getTelegramGroupChatInfo.ts
Original file line number Diff line number Diff line change
@@ -18,10 +18,20 @@ export const getTelegramGroupChatInfo = memoize(async (slug: string) => {
throw new Error("Element not found")
}

const [members, online] = element.textContent!.split(", ")
if (element.textContent!.includes("online")) {
const [members, online] = element.textContent!.split(", ")

const memberCount = +members.replace(" members", "").replace(" ", "")
const onlineCount = +online.replace(" online", "").replace(" ", "")
const memberCount = +members.replace(" members", "").replace(" ", "")
const onlineCount = +online.replace(" online", "").replace(" ", "")

return { memberCount, onlineCount }
return { memberCount, onlineCount }
} else if (element.textContent!.includes("subscribers")) {
const subscriberCount = +element
.textContent!.replace(" subscribers", "")
.replace(" ", "")

return { memberCount: subscriberCount, onlineCount: null }
}

throw new Error("Unknown type")
})
4 changes: 4 additions & 0 deletions src/functions/warpcast/getLastCastDateFromWarpcast.ts
Original file line number Diff line number Diff line change
@@ -12,5 +12,9 @@ export const getLastCastDateFromWarpcast = async (warpcastLink: string) => {

const response = await getWarpcastUserCasts(userInfo.result.user.fid)

if (response.result.casts.length === 0) {
return null
}

return new Date(response.result.casts[0].timestamp).toISOString()
}
2 changes: 1 addition & 1 deletion src/functions/warpcast/getWarpcastChannelCasts.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { ofetch } from "@/lib/ofetch"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"
import { z } from "zod"

/*
2 changes: 1 addition & 1 deletion src/functions/warpcast/getWarpcastChannelInfo.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { ofetch } from "@/lib/ofetch"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"
import { z } from "zod"

/*
2 changes: 1 addition & 1 deletion src/functions/warpcast/getWarpcastUserCasts.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { ofetch } from "@/lib/ofetch"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"
import { z } from "zod"

/*
8 changes: 5 additions & 3 deletions src/functions/warpcast/getWarpcastUserInfo.test.ts
Original file line number Diff line number Diff line change
@@ -2,11 +2,13 @@ import { getWarpcastUserInfo } from "@/functions/warpcast/getWarpcastUserInfo"
import { describe, expect, it } from "vitest"

describe("getWarpcastUserInfo function format tests", () => {
it("solana", async () => {
const userName = "solana"
it("Binance Coin", async () => {
const userName = "bnbchain"

const expected = 892762

const response = await getWarpcastUserInfo(userName)

expect(response.result.user.username).toBe(userName)
expect(response.result.user.fid).toBe(expected)
})
})
3 changes: 1 addition & 2 deletions src/functions/warpcast/getWarpcastUserInfo.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { ofetch } from "@/lib/ofetch"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"
import { z } from "zod"

/*
@@ -23,7 +23,6 @@ const getWrapcasetUserInfoSchema = z.object({
result: z.object({
user: z.object({
fid: z.number(),
username: z.string(),
followerCount: z.number(),
followingCount: z.number(),
}),
2 changes: 1 addition & 1 deletion src/functions/x/getLastPostDateFromX.ts
Original file line number Diff line number Diff line change
@@ -9,5 +9,5 @@ export const getLastPostDateFromX = async (xLink: string) => {
const { rest_id } = await getXUserInfo(screenName)
const { lastPostDate } = await getXUserPosts(rest_id)

return lastPostDate
return new Date(lastPostDate).toISOString()
}
2 changes: 1 addition & 1 deletion src/functions/x/getXUserInfo.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { getXHeaders } from "@/functions/x/getXHeaders"
import { ofetch } from "@/lib/ofetch"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"
import { z } from "zod"

/*
2 changes: 1 addition & 1 deletion src/functions/x/getXUserPosts.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { getXHeaders } from "@/functions/x/getXHeaders"
import { ofetch } from "@/lib/ofetch"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"
import { z } from "zod"

/*
44 changes: 44 additions & 0 deletions src/functions/youtube/getLastVideoDateFromYoutube.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
import { getLastVideoDateFromYoutube } from "@/functions/youtube/getLastVideoDateFromYoutube"
import { describe, expect, it } from "vitest"

describe("getLastVideoDateFromYoutube", () => {
it("ethereum", async () => {
const channelId = "UC6rYoXJ_3BbPyWx_GQDDRRQ"

const response = await getLastVideoDateFromYoutube(channelId)

expect(response).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/)
})

it.skip("algorand", async () => {
const channelId = "UCsda5E-IaXyUi8dzauyXypA"

const response = await getLastVideoDateFromYoutube(channelId)

expect(response).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/)
})

it.skip("bnb", async () => {
const channelId = "UCG9fZu6D4I83DStktBV0Ryw"

const response = await getLastVideoDateFromYoutube(channelId)

expect(response).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/)
})

it.skip("cardano", async () => {
const channelId = "UCbQ9vGfezru1YRI1zDCtTGg"

const response = await getLastVideoDateFromYoutube(channelId)

expect(response).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/)
})

it.skip("tron", async () => {
const channelId = "UC5OPOGRq02iK-0T9sKse_kA"

const response = await getLastVideoDateFromYoutube(channelId)

expect(response).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/)
})
})
48 changes: 40 additions & 8 deletions src/functions/youtube/getLastVideoDateFromYoutube.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,44 @@
import { getYoutubeChannelLastVideo } from "@/functions/youtube/getYoutubeChannelLastVideo"
import { getLastSegment } from "@/utils/getLastSegment"
import { env } from "@/lib/env"
import { ofetch } from "@/lib/ofetch"
import { memoize } from "@fxts/core"
import { z } from "zod"

export const getLastVideoDateFromYoutube = async (youtubeLink: string) => {
const channelId = getLastSegment(youtubeLink)
export const getLastVideoDateFromYoutube = memoize(
async (youtubeChannelId: string) => {
const playlistId = `${youtubeChannelId.slice(0, 1)}U${youtubeChannelId.slice(2)}`

const data = await getYoutubeChannelLastVideo(channelId)
const response = await ofetch<GetLastVideoDateFromYoutubeResponse>(
`https://www.googleapis.com/youtube/v3/playlistItems`,
{
query: {
part: "snippet",
playlistId,
maxResults: 1,
key: env.YOUTUBE_DATA_API_KEY,
},
},
)

const { publishedAt } = data.snippet
const data = getLastVideoDateFromYoutubeSchema.parse(response)

return publishedAt
}
const { publishedAt } = data.items[0].snippet

return new Date(publishedAt).toISOString()
},
)

const getLastVideoDateFromYoutubeSchema = z.object({
items: z
.array(
z.object({
snippet: z.object({
publishedAt: z.string(),
}),
}),
)
.min(1),
})

export type GetLastVideoDateFromYoutubeResponse = z.infer<
typeof getLastVideoDateFromYoutubeSchema
>
9 changes: 4 additions & 5 deletions src/functions/youtube/getSubscriberCountFromYoutube.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
import { getYoutubeChannelInfo } from "@/functions/youtube/getYoutubeChannelInfo"
import { getLastSegment } from "@/utils/getLastSegment"

export const getSubscriberCountFromYoutube = async (youtubeLink: string) => {
const channelId = getLastSegment(youtubeLink)

export const getSubscriberCountFromYoutube = async (
youtubeChannelId: string,
) => {
const {
statistics: { subscriberCount },
} = await getYoutubeChannelInfo(channelId)
} = await getYoutubeChannelInfo(youtubeChannelId)

return +subscriberCount
}
7 changes: 2 additions & 5 deletions src/functions/youtube/getVideoCountFromYoutube.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
import { getYoutubeChannelInfo } from "@/functions/youtube/getYoutubeChannelInfo"
import { getLastSegment } from "@/utils/getLastSegment"

export const getVideoCountFromYoutube = async (youtubeLink: string) => {
const channelId = getLastSegment(youtubeLink)

export const getVideoCountFromYoutube = async (youtubeChannelId: string) => {
const {
statistics: { videoCount },
} = await getYoutubeChannelInfo(channelId)
} = await getYoutubeChannelInfo(youtubeChannelId)

return +videoCount
}
9 changes: 4 additions & 5 deletions src/functions/youtube/getViewCountFromYoutube.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
import { getYoutubeChannelId } from "@/functions/youtube/getYoutubeChannelId"
import { getYoutubeChannelInfo } from "@/functions/youtube/getYoutubeChannelInfo"
import { getLastSegment } from "@/utils/getLastSegment"

export const getViewCountFromYoutube = async (youtubeLink: string) => {
const channelId = getLastSegment(youtubeLink)
import { youtubeUrlSchema } from "@/validators/youtube"

export const getViewCountFromYoutube = async (youtubeChannelId: string) => {
const {
statistics: { viewCount },
} = await getYoutubeChannelInfo(channelId)
} = await getYoutubeChannelInfo(youtubeChannelId)

return +viewCount
}
2 changes: 1 addition & 1 deletion src/functions/youtube/getYoutubeChannelFromWebsite.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { ofetch } from "@/lib/ofetch"
import { getDocument } from "@/utils/getDocument"
import { ofetch } from "ofetch"

export const getYoutubeChannelFromWebsite = async (url: string) => {
const html = await ofetch(url, { parseResponse: (txt) => txt })
45 changes: 45 additions & 0 deletions src/functions/youtube/getYoutubeChannelId.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import { getYoutubeChannelId } from "@/functions/youtube/getYoutubeChannelId"
import { describe, expect, it } from "vitest"

describe.skip("getYoutubeChannelId", () => {
it("/channel/<channel-id>", async () => {
const youtubeUrl =
"https://www.youtube.com/channel/UC6rYoXJ_3BbPyWx_GQDDRRQ"

const expected = "UC6rYoXJ_3BbPyWx_GQDDRRQ"

const response = await getYoutubeChannelId(youtubeUrl)

expect(response).toBe(expected)
})

it("/@<handle>", async () => {
const youtubeUrl = "https://youtube.com/@SolanaFndn"

const expected = "UC9AdQPUe4BdVJ8M9X7wxHUA"

const response = await getYoutubeChannelId(youtubeUrl)

expect(response).toBe(expected)
})

it("/c/<custom>", async () => {
const youtubeUrl = "https://youtube.com/c/Circlecryptofinance"

const expected = "UCzQMzwoT-Mj_TXggCLUT7Lw"

const response = await getYoutubeChannelId(youtubeUrl)

expect(response).toBe(expected)
})

it("/<custom>", async () => {
const youtubeUrl = "https://youtube.com/MakerDAO"

const expected = "UC4jqZlzQHUhzqf5rMd5ywTw"

const response = await getYoutubeChannelId(youtubeUrl)

expect(response).toBe(expected)
})
})
47 changes: 47 additions & 0 deletions src/functions/youtube/getYoutubeChannelId.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
import { env } from "@/lib/env"
import { ofetch } from "@/lib/ofetch"
import { getLastSegment } from "@/utils/getLastSegment"
import { memoize } from "@fxts/core"
import { z } from "zod"

export const getYoutubeChannelId = memoize(async (youtubeUrl: string) => {
const query = getLastSegment(youtubeUrl).replace("@", "")

const response = await ofetch<GetYoutubeChannelIdResponse>(
`https://www.googleapis.com/youtube/v3/search`,
{
query: {
type: `channel`,
part: `snippet`,
q: query,
maxResults: 1,
key: env.YOUTUBE_DATA_API_KEY,
},
},
)

const data = getYoutubeChannelIdSchema.parse(response)

return data.items[0].id.channelId
})

const getYoutubeChannelIdSchema = z.object({
kind: z.string(),
etag: z.string(),
items: z
.array(
z.object({
kind: z.string(),
etag: z.string(),
id: z.object({
kind: z.string(),
channelId: z.string(),
}),
}),
)
.min(1),
})

export type GetYoutubeChannelIdResponse = z.infer<
typeof getYoutubeChannelIdSchema
>
10 changes: 9 additions & 1 deletion src/functions/youtube/getYoutubeChannelInfo.test.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,20 @@
import { getYoutubeChannelInfo } from "@/functions/youtube/getYoutubeChannelInfo"
import { describe, expect, it } from "vitest"

describe("getYoutubeChannelInfo function format tests", () => {
describe("getYoutubeChannelInfo", () => {
it("solana", async () => {
const channelId = "UC9AdQPUe4BdVJ8M9X7wxHUA"

const response = await getYoutubeChannelInfo(channelId)

expect(response.id).toBe(channelId)
})

it.skip("near", async () => {
const channelId = "UCuKdIYVN8iE3fv8alyk1aMw"

const response = await getYoutubeChannelInfo(channelId)

expect(response.id).toBe(channelId)
})
})
2 changes: 1 addition & 1 deletion src/functions/youtube/getYoutubeChannelInfo.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { env } from "@/lib/env"
import { ofetch } from "@/lib/ofetch"
import { memoize } from "@fxts/core"
import { ofetch } from "ofetch"
import { z } from "zod"

/*
12 changes: 0 additions & 12 deletions src/functions/youtube/getYoutubeChannelLastVideo.test.ts

This file was deleted.

80 changes: 0 additions & 80 deletions src/functions/youtube/getYoutubeChannelLastVideo.ts

This file was deleted.

2 changes: 2 additions & 0 deletions src/lib/env.ts
Original file line number Diff line number Diff line change
@@ -3,6 +3,8 @@ import { z } from "zod"

export const REDDIT_CLIENT_ID = `0f6IA6dXKyjm3FV1hqrBSg`

export const COUNT_API_KEY = "eXRzYzpiZWk2emVleGFlMGFlaGFlMkhlZXcyZWVt"

export const PROXY_URL = `https://proxy.ysmdev.workers.dev`

export const env = createEnv({
2 changes: 2 additions & 0 deletions src/lib/getChainsSheet.test.ts
Original file line number Diff line number Diff line change
@@ -4,5 +4,7 @@ import { describe, it } from "vitest"
describe("sheet lib", () => {
it("getChainsSheet", async () => {
const response = await getChainsSheet()

console.log(JSON.stringify(response))
})
})
84 changes: 58 additions & 26 deletions src/lib/getChainsSheet.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { ofetch } from "ofetch"
import { ofetch } from "@/lib/ofetch"
import { map, pipe, toArray } from "@fxts/core"
import { z } from "zod"

export const getChainsSheet = async () => {
@@ -22,39 +23,70 @@ export const getChainsSheet = async () => {
const dataRows = rows.slice(1)

const jsonArray = dataRows.map((row) => {
const obj: { [key: string]: string } = {}
const obj: { [key: string]: string | null } = {}
headers.forEach((header, index) => {
obj[header] = row[index].replace(/^"|"$/g, "").trim() || "" // 값이 없으면 빈 문자열
obj[header] = row[index].replace(/^"|"$/g, "").trim() || null // 값이 없으면 빈 문자열
})
return obj
})

return getChainsSheetSchema.parse(jsonArray)
const data = getChainsSheetSchema.parse(jsonArray)

return pipe(
data,
map((v) => ({
name: v.Name,
slug: v["Key (slug)"],
type: v.type,
symbol: v.Symbol,
founder: v.Founder,
founded: v.Founded,
firstBlock: v.FirstBlock,
hq: v.HQ,
isMainnet: v.isMainnet,
websiteUrl: v.Website,
gitHubOrganizationUrl: v.GitHub,
gitHubRepositoryUrl: v["GitHub main repo"],
xUrl: v["𝕏"],
linkedInUrl: v.LinkedIn,
youtubeUrl: v.Youtube,
youtubeChannelId: v["Youtube Channel ID"],
telegramUrl: v.Telegram,
discordUrl: v.Discord,
redditUrl: v.Reddit,
warpcastProfileUrl: v["Farcaster Profile"],
warpcastChannelUrl: v["Farcaster Channel"],
blogUrl: v.Blog,
npmUrl: v.Npm,
})),
toArray,
)
}

const getChainsSheetSchema = z.array(
z.object({
Name: z.string().optional(),
Key: z.string().optional(),
type: z.string().optional(),
Symbol: z.string().optional(),
Founder: z.string().optional(),
Founded: z.string().optional(),
FirstBlock: z.string().optional(),
HQ: z.string().optional(),
isMainnet: z.string().optional(),
Website: z.string().optional(),
GitHub: z.string().optional(),
["GitHub main repo"]: z.string().optional(),
X: z.string().optional(),
LinkedIn: z.string().optional(),
Youtube: z.string().optional(),
Telegram: z.string().optional(),
Discord: z.string().optional(),
Reddit: z.string().optional(),
["Farcaster Profile"]: z.string().optional(),
["Farcaster Channel"]: z.string().optional(),
Blog: z.string().optional(),
["JS Library"]: z.string().optional(),
Name: z.string().nullable(),
["Key (slug)"]: z.string().nullable(),
type: z.string().nullable(),
Symbol: z.string().nullable(),
Founder: z.string().nullable(),
Founded: z.string().nullable(),
FirstBlock: z.string().nullable(),
HQ: z.string().nullable(),
isMainnet: z.string().nullable(),
Website: z.string().nullable(),
GitHub: z.string().nullable(),
["GitHub main repo"]: z.string().nullable(),
𝕏: z.string().nullable(),
LinkedIn: z.string().nullable(),
Youtube: z.string().nullable(),
["Youtube Channel ID"]: z.string().nullable(),
Telegram: z.string().nullable(),
Discord: z.string().nullable(),
Reddit: z.string().nullable(),
["Farcaster Profile"]: z.string().nullable(),
["Farcaster Channel"]: z.string().nullable(),
Blog: z.string().nullable(),
Npm: z.string().nullable(),
}),
)
7 changes: 6 additions & 1 deletion src/lib/ofetch.ts
Original file line number Diff line number Diff line change
@@ -8,16 +8,21 @@ export const ofetch = async <T = any, R extends ResponseType = "json">(
) => {
const response = await fetch.raw<T, R>(request, {
...options,
ignoreResponseError: true,
})

if (response.status === 403 || response.status === 429) {
console.log("Rate limited")
console.log("Rate limited", request)
const data = await fetch<T, R>(`${PROXY_URL}/${request}`, {
...options,
})

return data
}

if (response.status === 404) {
return null
}

return response._data
}
447 changes: 262 additions & 185 deletions src/main.ts

Large diffs are not rendered by default.

20 changes: 20 additions & 0 deletions src/validators/youtube.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
import { urlSchema } from "@/validators/urlSchema"
import { z } from "zod"

export const youtubeUrlSchema = z.custom<string>((value) => {
const validUrl = urlSchema.parse(value)

const { pathname } = new URL(validUrl)

const channelRegex = /^\/channel\/([A-Za-z0-9_-]+)$/
const handleRegex = /^\/@([\w-]+)$/
const brandRegex = /^\/c\/([\w-]+)$/
const customRegex = /^\/([\w-]+)$/

return (
channelRegex.test(pathname) ||
handleRegex.test(pathname) ||
brandRegex.test(pathname) ||
customRegex.test(pathname)
)
})

0 comments on commit 75baf2c

Please sign in to comment.